package com.itcast.spark.baseFeaturation

import org.apache.spark.ml.feature.{IndexToString, OneHotEncoder, OneHotEncoderEstimator, OneHotEncoderModel, StringIndexer, StringIndexerModel}
import org.apache.spark.{SparkConf, SparkContext}
import org.apache.spark.sql.{DataFrame, SparkSession}

/**
 * DESC:目的是用于类别值属性的数值化
 * 1-stringtoindexer
 * 2-indexertostring
 * 3-onehotencoder
 */
object _01LabelencoderTest {
  def main(args: Array[String]): Unit = {
    //这里是准备环境
    val conf: SparkConf = new SparkConf().setAppName("_04RandomNumber").setMaster("local[*]")
    val spark: SparkSession = SparkSession.builder().config(conf).getOrCreate()
    val sc: SparkContext = spark.sparkContext
    sc.setLogLevel("WARN")
    //这里是准备数据
    val df = spark.createDataFrame(
      Seq((0, "a"), (1, "b"), (2, "c"), (3, "a"), (4, "a"), (5, "c"))
    ).toDF("id", "category")
    df.show()
    //1-stringtoindexer
    //.setInputCol("category") 设置输入的列的schema的信息，输入的列一定是来源于你想处理的列
    //.setOutputCol("index") 设置输出列的信息
    val indexer: StringIndexer = new StringIndexer().setInputCol("category").setOutputCol("index")
    val indexerModel: StringIndexerModel = indexer.fit(df)
    val indexResult: DataFrame = indexerModel.transform(df)
    /*    indexResult.show()
        +---+--------+-----+
        | id|category|index|
        +---+--------+-----+
        |  0|       a|  0.0|
        |  1|       b|  2.0|
        |  2|       c|  1.0|
        |  3|       a|  0.0|
        |  4|       a|  0.0|
        |  5|       c|  1.0|
        +---+--------+-----+*/
    //2-indexertostring----在预测过程中预测的结果是0-1，对应0是男性-1是女性
    //.setLabels(indexerModel.labels) 该参数是需要设置对应关系0-男生 1-男生
    val indexToString: IndexToString = new IndexToString().setInputCol("index").setOutputCol("beforeIndex")
      .setLabels(indexerModel.labels)
    val stringDF: DataFrame = indexToString.transform(indexResult)
    stringDF
      .show()
    //3-onehotencoder
    //这里onehot编码需要传入的数值一定是数值型的数据，，API的要求
    val encoder: OneHotEncoder = new OneHotEncoder().setInputCol("index").setOutputCol("oheResult").setDropLast(false)
    encoder.transform(stringDF).show()

    val estimator: OneHotEncoderEstimator = new OneHotEncoderEstimator().setInputCols(Array("index")).setOutputCols(Array("oheResult1")).setDropLast(false)
    estimator.fit(stringDF).transform(stringDF).show()
    /*+---+--------+-----+-----------+-------------+
    | id|category|index|beforeIndex|   oheResult1|
    +---+--------+-----+-----------+-------------+
    |  0|       a|  0.0|          a|(3,[0],[1.0])|
    |  1|       b|  2.0|          b|(3,[2],[1.0])|
    |  2|       c|  1.0|          c|(3,[1],[1.0])|
    |  3|       a|  0.0|          a|(3,[0],[1.0])|
    |  4|       a|  0.0|          a|(3,[0],[1.0])|
    |  5|       c|  1.0|          c|(3,[1],[1.0])|
    +---+--------+-----+-----------+-------------+*/
  }
}
